{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.5"
    },
    "colab": {
      "name": "2_Build_a_processor_for_your_own_dataset.ipynb",
      "provenance": []
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0fedf11acab6486d8c734b79290d6875": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_daced229491c441ea69ffe85c36867a5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c39407be900946029a6103d448fb8fb2",
              "IPY_MODEL_c990a7513c1a4a3398eaef7699fcda7b"
            ]
          }
        },
        "daced229491c441ea69ffe85c36867a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c39407be900946029a6103d448fb8fb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a421ec39b3b944e6bd444ef6019a5808",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_29c9b4c052f5482097b678a2b74bb626"
          }
        },
        "c990a7513c1a4a3398eaef7699fcda7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7dcf072091ac4753a9d2a8f41d5ecc82",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 3.51MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2be8f3cd6a81443a8d74f08a98c2b23b"
          }
        },
        "a421ec39b3b944e6bd444ef6019a5808": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "29c9b4c052f5482097b678a2b74bb626": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7dcf072091ac4753a9d2a8f41d5ecc82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2be8f3cd6a81443a8d74f08a98c2b23b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6ZQslRT0Fqy-",
        "colab_type": "text"
      },
      "source": [
        "## FARM: Use your own dataset\n",
        "    \n",
        "In Tutorial 1 you already learned about the major building blocks.\n",
        "In this tutorial, you will see how to use FARM with your own dataset."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "usDHxZCBFxgU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "1303e268-39b2-4fa5-a5bf-999965638dc7"
      },
      "source": [
        "!git clone https://github.com/deepset-ai/FARM.git\n",
        "import os\n",
        "os.chdir(\"FARM\")\n",
        "!pip install -r requirements.txt\n",
        "!pip install --editable ."
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'FARM'...\n",
            "remote: Enumerating objects: 54, done.\u001b[K\n",
            "remote: Counting objects: 100% (54/54), done.\u001b[K\n",
            "remote: Compressing objects: 100% (41/41), done.\u001b[K\n",
            "remote: Total 4472 (delta 26), reused 29 (delta 12), pack-reused 4418\u001b[K\n",
            "Receiving objects: 100% (4472/4472), 65.08 MiB | 27.22 MiB/s, done.\n",
            "Resolving deltas: 100% (3290/3290), done.\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 2)) (46.1.3)\n",
            "Requirement already satisfied: wheel in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 3)) (0.34.2)\n",
            "Collecting torch==1.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/24/19/4804aea17cd136f1705a5e98a00618cb8f6ccc375ad8bfa437408e09d058/torch-1.4.0-cp36-cp36m-manylinux1_x86_64.whl (753.4MB)\n",
            "\u001b[K     |████████████████████████████████| 753.4MB 21kB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 7)) (4.38.0)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 9)) (1.12.47)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 11)) (2.23.0)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 13)) (1.4.1)\n",
            "Requirement already satisfied: sklearn in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 14)) (0.0)\n",
            "Collecting seqeval\n",
            "  Downloading https://files.pythonhosted.org/packages/34/91/068aca8d60ce56dd9ba4506850e876aba5e66a6f2f29aa223224b50df0de/seqeval-0.0.12.tar.gz\n",
            "Collecting mlflow==1.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/01/ec/8c9448968d4662e8354b9c3a62e635f8929ed507a45af3d9fdb84be51270/mlflow-1.0.0-py3-none-any.whl (47.7MB)\n",
            "\u001b[K     |████████████████████████████████| 47.7MB 57kB/s \n",
            "\u001b[?25hCollecting transformers==2.7.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/37/ba/dda44bbf35b071441635708a3dd568a5ca6bf29f77389f7c7c6818ae9498/transformers-2.7.0-py3-none-any.whl (544kB)\n",
            "\u001b[K     |████████████████████████████████| 552kB 57.6MB/s \n",
            "\u001b[?25hCollecting dotmap==1.3.0\n",
            "  Downloading https://files.pythonhosted.org/packages/fa/eb/ee5f0358a9e0ede90308d8f34e697e122f191c2702dc4f614eca7770b1eb/dotmap-1.3.0-py3-none-any.whl\n",
            "Collecting Werkzeug==0.16.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c2/e4/a859d2fe516f466642fa5c6054fd9646271f9da26b0cac0d2f37fc858c8f/Werkzeug-0.16.1-py2.py3-none-any.whl (327kB)\n",
            "\u001b[K     |████████████████████████████████| 327kB 62.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: flask in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 24)) (1.1.2)\n",
            "Collecting flask-restplus\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c2/a6/b17c848771f96ad039ad9e3ea275e842a16c39c4f3eb9f60ee330b20b6c2/flask_restplus-0.13.0-py2.py3-none-any.whl (2.5MB)\n",
            "\u001b[K     |████████████████████████████████| 2.5MB 53.5MB/s \n",
            "\u001b[?25hCollecting flask-cors\n",
            "  Downloading https://files.pythonhosted.org/packages/78/38/e68b11daa5d613e3a91e4bf3da76c94ac9ee0d9cd515af9c1ab80d36f709/Flask_Cors-3.0.8-py2.py3-none-any.whl\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 27)) (0.3.1.1)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->-r requirements.txt (line 9)) (0.3.3)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->-r requirements.txt (line 9)) (0.9.5)\n",
            "Requirement already satisfied: botocore<1.16.0,>=1.15.47 in /usr/local/lib/python3.6/dist-packages (from boto3->-r requirements.txt (line 9)) (1.15.47)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->-r requirements.txt (line 11)) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->-r requirements.txt (line 11)) (2020.4.5.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->-r requirements.txt (line 11)) (2.9)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->-r requirements.txt (line 11)) (3.0.4)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy>=1.3.2->-r requirements.txt (line 13)) (1.18.3)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sklearn->-r requirements.txt (line 14)) (0.22.2.post1)\n",
            "Requirement already satisfied: Keras>=2.2.4 in /usr/local/lib/python3.6/dist-packages (from seqeval->-r requirements.txt (line 16)) (2.3.1)\n",
            "Requirement already satisfied: sqlparse in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->-r requirements.txt (line 17)) (0.3.1)\n",
            "Collecting alembic\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/60/1e/cabc75a189de0fbb2841d0975243e59bde8b7822bacbb95008ac6fe9ad47/alembic-1.4.2.tar.gz (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 50.6MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting querystring-parser\n",
            "  Downloading https://files.pythonhosted.org/packages/4a/fa/f54f5662e0eababf0c49e92fd94bf178888562c0e7b677c8941bbbcd1bd6/querystring_parser-1.2.4.tar.gz\n",
            "Collecting databricks-cli>=0.8.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/49/d1/fe0ba3d5c2b4b76ec035aa243bbc2fd0d60607a391f192ebe1656e17a4e2/databricks-cli-0.10.0.tar.gz (45kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 9.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->-r requirements.txt (line 17)) (3.13)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->-r requirements.txt (line 17)) (7.1.2)\n",
            "Collecting gitpython>=2.1.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/44/33/917e6fde1cad13daa7053f39b7c8af3be287314f75f1b1ea8d3fe37a8571/GitPython-3.1.2-py3-none-any.whl (451kB)\n",
            "\u001b[K     |████████████████████████████████| 460kB 53.3MB/s \n",
            "\u001b[?25hCollecting docker>=3.6.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/58/74/379a9d30b1620def158c40b88c43e01c1936a287ebb97afab0699c601c57/docker-4.2.0-py2.py3-none-any.whl (143kB)\n",
            "\u001b[K     |████████████████████████████████| 153kB 64.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->-r requirements.txt (line 17)) (2.8.1)\n",
            "Requirement already satisfied: sqlalchemy in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->-r requirements.txt (line 17)) (1.3.16)\n",
            "Collecting gunicorn\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/69/ca/926f7cd3a2014b16870086b2d0fdc84a9e49473c68a8dff8b57f7c156f43/gunicorn-20.0.4-py2.py3-none-any.whl (77kB)\n",
            "\u001b[K     |████████████████████████████████| 81kB 13.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->-r requirements.txt (line 17)) (3.10.0)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->-r requirements.txt (line 17)) (1.3.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->-r requirements.txt (line 17)) (1.0.3)\n",
            "Collecting simplejson\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/98/87/a7b98aa9256c8843f92878966dc3d8d914c14aad97e2c5ce4798d5743e07/simplejson-3.17.0.tar.gz (83kB)\n",
            "\u001b[K     |████████████████████████████████| 92kB 15.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->-r requirements.txt (line 17)) (1.12.0)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->-r requirements.txt (line 17)) (0.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==2.7.0->-r requirements.txt (line 19)) (3.0.12)\n",
            "Collecting tokenizers==0.5.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/3f/73c881ea4723e43c1e9acf317cf407fab3a278daab3a69c98dcac511c04f/tokenizers-0.5.2-cp36-cp36m-manylinux1_x86_64.whl (3.7MB)\n",
            "\u001b[K     |████████████████████████████████| 3.7MB 48.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==2.7.0->-r requirements.txt (line 19)) (2019.12.20)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers==2.7.0->-r requirements.txt (line 19)) (0.7)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 51.6MB/s \n",
            "\u001b[?25hCollecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/98/2c/8df20f3ac6c22ac224fff307ebc102818206c53fc454ecd37d8ac2060df5/sentencepiece-0.1.86-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 51.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: itsdangerous>=0.24 in /usr/local/lib/python3.6/dist-packages (from flask->-r requirements.txt (line 24)) (1.1.0)\n",
            "Requirement already satisfied: Jinja2>=2.10.1 in /usr/local/lib/python3.6/dist-packages (from flask->-r requirements.txt (line 24)) (2.11.2)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.6/dist-packages (from flask-restplus->-r requirements.txt (line 25)) (2.6.0)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.6/dist-packages (from flask-restplus->-r requirements.txt (line 25)) (2018.9)\n",
            "Collecting aniso8601>=0.82\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/eb/e4/787e104b58eadc1a710738d4e418d7e599e4e778e52cb8e5d5ef6ddd5833/aniso8601-8.0.0-py2.py3-none-any.whl (43kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 8.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.47->boto3->-r requirements.txt (line 9)) (0.15.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn->-r requirements.txt (line 14)) (0.14.1)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->-r requirements.txt (line 16)) (1.1.0)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->-r requirements.txt (line 16)) (2.10.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->-r requirements.txt (line 16)) (1.0.8)\n",
            "Collecting Mako\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/50/78/f6ade1e18aebda570eed33b7c534378d9659351cadce2fcbc7b31be5f615/Mako-1.1.2-py2.py3-none-any.whl (75kB)\n",
            "\u001b[K     |████████████████████████████████| 81kB 13.6MB/s \n",
            "\u001b[?25hCollecting python-editor>=0.3\n",
            "  Downloading https://files.pythonhosted.org/packages/c6/d3/201fc3abe391bbae6606e6f1d598c15d367033332bd54352b12f35513717/python_editor-1.0.4-py3-none-any.whl\n",
            "Requirement already satisfied: tabulate>=0.7.7 in /usr/local/lib/python3.6/dist-packages (from databricks-cli>=0.8.0->mlflow==1.0.0->-r requirements.txt (line 17)) (0.8.7)\n",
            "Collecting configparser>=0.3.5\n",
            "  Downloading https://files.pythonhosted.org/packages/4b/6b/01baa293090240cf0562cc5eccb69c6f5006282127f2b846fad011305c79/configparser-5.0.0-py3-none-any.whl\n",
            "Collecting gitdb<5,>=4.0.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/48/11/d1800bca0a3bae820b84b7d813ad1eff15a48a64caea9c823fc8c1b119e8/gitdb-4.0.5-py3-none-any.whl (63kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 11.6MB/s \n",
            "\u001b[?25hCollecting websocket-client>=0.32.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4c/5f/f61b420143ed1c8dc69f9eaec5ff1ac36109d52c80de49d66e0c36c3dfdf/websocket_client-0.57.0-py2.py3-none-any.whl (200kB)\n",
            "\u001b[K     |████████████████████████████████| 204kB 64.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from Jinja2>=2.10.1->flask->-r requirements.txt (line 24)) (1.1.1)\n",
            "Collecting smmap<4,>=3.0.1\n",
            "  Downloading https://files.pythonhosted.org/packages/b0/9a/4d409a6234eb940e6a78dfdfc66156e7522262f5f2fecca07dc55915952d/smmap-3.0.4-py2.py3-none-any.whl\n",
            "Building wheels for collected packages: alembic\n",
            "  Building wheel for alembic (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for alembic: filename=alembic-1.4.2-cp36-none-any.whl size=159543 sha256=e0792381ca2f31112fbb5f77c061740ef7a6e71b838cba1a69c0b056b2d88bc7\n",
            "  Stored in directory: /root/.cache/pip/wheels/1f/04/83/76023f7a4c14688c0b5c2682a96392cfdd3ee4449eaaa287ef\n",
            "Successfully built alembic\n",
            "Building wheels for collected packages: seqeval, querystring-parser, databricks-cli, simplejson, sacremoses\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-0.0.12-cp36-none-any.whl size=7424 sha256=ff2b06d2eda263d23f23645247be90a26d774afca9a1c639ad94bcb3cca9c5c3\n",
            "  Stored in directory: /root/.cache/pip/wheels/4f/32/0a/df3b340a82583566975377d65e724895b3fad101a3fb729f68\n",
            "  Building wheel for querystring-parser (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for querystring-parser: filename=querystring_parser-1.2.4-cp36-none-any.whl size=7079 sha256=3f93f44a1cf7b33b9b367f28200f975c16de7e9440def52a46677439f094749a\n",
            "  Stored in directory: /root/.cache/pip/wheels/1e/41/34/23ebf5d1089a9aed847951e0ee375426eb4ad0a7079d88d41e\n",
            "  Building wheel for databricks-cli (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for databricks-cli: filename=databricks_cli-0.10.0-cp36-none-any.whl size=84285 sha256=fb4e086697e1bc05fbe93e7cda8600cc43230501043aefea6a05a5f2b1add1ae\n",
            "  Stored in directory: /root/.cache/pip/wheels/1e/e5/2d/a19c0bfd38005176063f130d72de17cb3d2d32c0ee384e7493\n",
            "  Building wheel for simplejson (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for simplejson: filename=simplejson-3.17.0-cp36-cp36m-linux_x86_64.whl size=114209 sha256=a656cf54632c82427fbe3e7d0b6af3fd78044fa920e98c485e76bf2e347e7ec9\n",
            "  Stored in directory: /root/.cache/pip/wheels/86/c0/83/dcd0339abb2640544bb8e0938aab2d069cef55e5647ce6e097\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=e8bd50c5229f337027342b0d8a43cddb50ef5ffd5acd090a5cf0a62d9fc6c7d8\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built seqeval querystring-parser databricks-cli simplejson sacremoses\n",
            "\u001b[31mERROR: torchvision 0.6.0+cu101 has requirement torch==1.5.0, but you'll have torch 1.4.0 which is incompatible.\u001b[0m\n",
            "Installing collected packages: torch, seqeval, Mako, python-editor, alembic, querystring-parser, configparser, databricks-cli, smmap, gitdb, gitpython, websocket-client, docker, gunicorn, simplejson, mlflow, tokenizers, sacremoses, sentencepiece, transformers, dotmap, Werkzeug, aniso8601, flask-restplus, flask-cors\n",
            "  Found existing installation: torch 1.5.0+cu101\n",
            "    Uninstalling torch-1.5.0+cu101:\n",
            "      Successfully uninstalled torch-1.5.0+cu101\n",
            "  Found existing installation: Werkzeug 1.0.1\n",
            "    Uninstalling Werkzeug-1.0.1:\n",
            "      Successfully uninstalled Werkzeug-1.0.1\n",
            "Successfully installed Mako-1.1.2 Werkzeug-0.16.1 alembic-1.4.2 aniso8601-8.0.0 configparser-5.0.0 databricks-cli-0.10.0 docker-4.2.0 dotmap-1.3.0 flask-cors-3.0.8 flask-restplus-0.13.0 gitdb-4.0.5 gitpython-3.1.2 gunicorn-20.0.4 mlflow-1.0.0 python-editor-1.0.4 querystring-parser-1.2.4 sacremoses-0.0.43 sentencepiece-0.1.86 seqeval-0.0.12 simplejson-3.17.0 smmap-3.0.4 tokenizers-0.5.2 torch-1.4.0 transformers-2.7.0 websocket-client-0.57.0\n",
            "Obtaining file:///content/FARM\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (46.1.3)\n",
            "Requirement already satisfied: wheel in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (0.34.2)\n",
            "Requirement already satisfied: torch==1.4.0 in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (1.4.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (4.38.0)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (1.12.47)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (2.23.0)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (1.4.1)\n",
            "Requirement already satisfied: sklearn in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (0.0)\n",
            "Requirement already satisfied: seqeval in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (0.0.12)\n",
            "Requirement already satisfied: mlflow==1.0.0 in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (1.0.0)\n",
            "Requirement already satisfied: transformers==2.7.0 in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (2.7.0)\n",
            "Requirement already satisfied: dotmap==1.3.0 in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (1.3.0)\n",
            "Requirement already satisfied: Werkzeug==0.16.1 in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (0.16.1)\n",
            "Requirement already satisfied: flask in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (1.1.2)\n",
            "Requirement already satisfied: flask-restplus in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (0.13.0)\n",
            "Requirement already satisfied: flask-cors in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (3.0.8)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.6/dist-packages (from farm==0.4.3) (0.3.1.1)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->farm==0.4.3) (0.3.3)\n",
            "Requirement already satisfied: botocore<1.16.0,>=1.15.47 in /usr/local/lib/python3.6/dist-packages (from boto3->farm==0.4.3) (1.15.47)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->farm==0.4.3) (0.9.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->farm==0.4.3) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->farm==0.4.3) (2020.4.5.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->farm==0.4.3) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->farm==0.4.3) (2.9)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy>=1.3.2->farm==0.4.3) (1.18.3)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sklearn->farm==0.4.3) (0.22.2.post1)\n",
            "Requirement already satisfied: Keras>=2.2.4 in /usr/local/lib/python3.6/dist-packages (from seqeval->farm==0.4.3) (2.3.1)\n",
            "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (3.10.0)\n",
            "Requirement already satisfied: docker>=3.6.0 in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (4.2.0)\n",
            "Requirement already satisfied: gitpython>=2.1.0 in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (3.1.2)\n",
            "Requirement already satisfied: querystring-parser in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (1.2.4)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (1.3.0)\n",
            "Requirement already satisfied: sqlparse in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (0.3.1)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (0.3)\n",
            "Requirement already satisfied: simplejson in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (3.17.0)\n",
            "Requirement already satisfied: gunicorn in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (20.0.4)\n",
            "Requirement already satisfied: sqlalchemy in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (1.3.16)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (3.13)\n",
            "Requirement already satisfied: databricks-cli>=0.8.0 in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (0.10.0)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (7.1.2)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (2.8.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (1.0.3)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (1.12.0)\n",
            "Requirement already satisfied: alembic in /usr/local/lib/python3.6/dist-packages (from mlflow==1.0.0->farm==0.4.3) (1.4.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==2.7.0->farm==0.4.3) (2019.12.20)\n",
            "Requirement already satisfied: tokenizers==0.5.2 in /usr/local/lib/python3.6/dist-packages (from transformers==2.7.0->farm==0.4.3) (0.5.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==2.7.0->farm==0.4.3) (3.0.12)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers==2.7.0->farm==0.4.3) (0.0.43)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers==2.7.0->farm==0.4.3) (0.7)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers==2.7.0->farm==0.4.3) (0.1.86)\n",
            "Requirement already satisfied: itsdangerous>=0.24 in /usr/local/lib/python3.6/dist-packages (from flask->farm==0.4.3) (1.1.0)\n",
            "Requirement already satisfied: Jinja2>=2.10.1 in /usr/local/lib/python3.6/dist-packages (from flask->farm==0.4.3) (2.11.2)\n",
            "Requirement already satisfied: aniso8601>=0.82 in /usr/local/lib/python3.6/dist-packages (from flask-restplus->farm==0.4.3) (8.0.0)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.6/dist-packages (from flask-restplus->farm==0.4.3) (2018.9)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.6/dist-packages (from flask-restplus->farm==0.4.3) (2.6.0)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.47->boto3->farm==0.4.3) (0.15.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn->farm==0.4.3) (0.14.1)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->farm==0.4.3) (2.10.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->farm==0.4.3) (1.0.8)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from Keras>=2.2.4->seqeval->farm==0.4.3) (1.1.0)\n",
            "Requirement already satisfied: websocket-client>=0.32.0 in /usr/local/lib/python3.6/dist-packages (from docker>=3.6.0->mlflow==1.0.0->farm==0.4.3) (0.57.0)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.6/dist-packages (from gitpython>=2.1.0->mlflow==1.0.0->farm==0.4.3) (4.0.5)\n",
            "Requirement already satisfied: tabulate>=0.7.7 in /usr/local/lib/python3.6/dist-packages (from databricks-cli>=0.8.0->mlflow==1.0.0->farm==0.4.3) (0.8.7)\n",
            "Requirement already satisfied: configparser>=0.3.5 in /usr/local/lib/python3.6/dist-packages (from databricks-cli>=0.8.0->mlflow==1.0.0->farm==0.4.3) (5.0.0)\n",
            "Requirement already satisfied: Mako in /usr/local/lib/python3.6/dist-packages (from alembic->mlflow==1.0.0->farm==0.4.3) (1.1.2)\n",
            "Requirement already satisfied: python-editor>=0.3 in /usr/local/lib/python3.6/dist-packages (from alembic->mlflow==1.0.0->farm==0.4.3) (1.0.4)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from Jinja2>=2.10.1->flask->farm==0.4.3) (1.1.1)\n",
            "Requirement already satisfied: smmap<4,>=3.0.1 in /usr/local/lib/python3.6/dist-packages (from gitdb<5,>=4.0.1->gitpython>=2.1.0->mlflow==1.0.0->farm==0.4.3) (3.0.4)\n",
            "Installing collected packages: farm\n",
            "  Running setup.py develop for farm\n",
            "Successfully installed farm\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x2N7NwpuFqzD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "e2327856-63cf-42b6-f1a6-c930b2f00bd0"
      },
      "source": [
        "# Let's start by adjust the working directory so that it is the root of the repository\n",
        "# This should be run just once.\n",
        "\n",
        "import os\n",
        "os.chdir('../')\n",
        "print(\"Current working directory is {}\".format(os.getcwd()))"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Current working directory is /content/FARM\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ESbe1stqFqzT",
        "colab_type": "text"
      },
      "source": [
        "# 1) How a Processor works"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KlrvOVB4FqzV",
        "colab_type": "text"
      },
      "source": [
        "### Architecture\n",
        "The Processor converts a <b>raw input (e.g File) into a Pytorch dataset</b>.   \n",
        "For using an own dataset we need to adjust this Processor.\n",
        "\n",
        "<img src=\"https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/data_silo_no_bg.jpg\" width=\"400\" height=\"400\" align=\"left\"/>\n",
        "<br/><br/>\n",
        "<br/><br/>\n",
        "<br/><br/>\n",
        "<br/><br/>\n",
        "<br/><br/>\n",
        "<br/><br/>\n",
        "<br/><br/>\n",
        "\n",
        "​\n",
        "### Main Conversion Stages \n",
        "1. Read from file / raw input \n",
        "2. Create samples\n",
        "3. Featurize samples\n",
        "4. Create PyTorch Dataset\n",
        "\n",
        "### Functions to implement\n",
        "1. file\\_to_dicts()\n",
        "2. \\_dict_to_samples()\n",
        "3. \\_sample_to_features()  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "11IJzSOtFqzW",
        "colab_type": "text"
      },
      "source": [
        "## Example: TextClassificationProcessor"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PjwdbEnOFqzZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "ef59991f-f22e-4821-dd43-659341d841a4"
      },
      "source": [
        "from farm.data_handler.processor import *\n",
        "from farm.data_handler.samples import Sample\n",
        "from farm.modeling.tokenization import Tokenizer, tokenize_with_metadata\n",
        "\n",
        "import os\n",
        "\n",
        "class TextClassificationProcessor(Processor):\n",
        "    \"\"\"\n",
        "    Used to handle the text classification datasets that come in tabular format (CSV, TSV, etc.)\n",
        "    \"\"\"\n",
        "    def __init__(\n",
        "        self,\n",
        "        tokenizer,\n",
        "        max_seq_len,\n",
        "        data_dir,\n",
        "        label_list=None,\n",
        "        metric=None,\n",
        "        train_filename=\"train.tsv\",\n",
        "        dev_filename=None,\n",
        "        test_filename=\"test.tsv\",\n",
        "        dev_split=0.1,\n",
        "        delimiter=\"\\t\",\n",
        "        quote_char=\"'\",\n",
        "        skiprows=None,\n",
        "        label_column_name=\"label\",\n",
        "        multilabel=False,\n",
        "        header=0,\n",
        "        proxies=None,\n",
        "        max_samples=None,\n",
        "        **kwargs\n",
        "    ):\n",
        "        #TODO If an arg is misspelt, e.g. metrics, it will be swallowed silently by kwargs\n",
        "\n",
        "        # Custom processor attributes\n",
        "        self.delimiter = delimiter\n",
        "        self.quote_char = quote_char\n",
        "        self.skiprows = skiprows\n",
        "        self.header = header\n",
        "        self.max_samples = max_samples\n",
        "\n",
        "        super(TextClassificationProcessor, self).__init__(\n",
        "            tokenizer=tokenizer,\n",
        "            max_seq_len=max_seq_len,\n",
        "            train_filename=train_filename,\n",
        "            dev_filename=dev_filename,\n",
        "            test_filename=test_filename,\n",
        "            dev_split=dev_split,\n",
        "            data_dir=data_dir,\n",
        "            tasks={},\n",
        "            proxies=proxies,\n",
        "\n",
        "        )\n",
        "        if metric and label_list:\n",
        "            if multilabel:\n",
        "                task_type = \"multilabel_classification\"\n",
        "            else:\n",
        "                task_type = \"classification\"\n",
        "            self.add_task(name=\"text_classification\",\n",
        "                          metric=metric,\n",
        "                          label_list=label_list,\n",
        "                          label_column_name=label_column_name,\n",
        "                          task_type=task_type)\n",
        "        else:\n",
        "            logger.info(\"Initialized processor without tasks. Supply `metric` and `label_list` to the constructor for \"\n",
        "                        \"using the default task or add a custom task later via processor.add_task()\")\n",
        "\n",
        "    def file_to_dicts(self, file: str) -> [dict]:\n",
        "        column_mapping = {task[\"label_column_name\"]: task[\"label_name\"] for task in self.tasks.values()}\n",
        "        dicts = read_tsv(\n",
        "            filename=file,\n",
        "            delimiter=self.delimiter,\n",
        "            skiprows=self.skiprows,\n",
        "            quotechar=self.quote_char,\n",
        "            rename_columns=column_mapping,\n",
        "            header=self.header,\n",
        "            proxies=self.proxies,\n",
        "            max_samples=self.max_samples\n",
        "            )\n",
        "\n",
        "        return dicts\n",
        "\n",
        "    def _dict_to_samples(self, dictionary: dict, **kwargs) -> [Sample]:\n",
        "        # this tokenization also stores offsets and a start_of_word mask\n",
        "        text = dictionary[\"text\"]\n",
        "        tokenized = tokenize_with_metadata(text, self.tokenizer)\n",
        "        if len(tokenized[\"tokens\"]) == 0:\n",
        "            logger.warning(f\"The following text could not be tokenized, likely because it contains a character that the tokenizer does not recognize: {text}\")\n",
        "            return []\n",
        "        # truncate tokens, offsets and start_of_word to max_seq_len that can be handled by the model\n",
        "        for seq_name in tokenized.keys():\n",
        "            tokenized[seq_name], _, _ = truncate_sequences(seq_a=tokenized[seq_name], seq_b=None, tokenizer=self.tokenizer,\n",
        "                                                max_seq_len=self.max_seq_len)\n",
        "        return [Sample(id=None, clear_text=dictionary, tokenized=tokenized)]\n",
        "\n",
        "    def _sample_to_features(self, sample) -> dict:\n",
        "        features = sample_to_features_text(\n",
        "            sample=sample,\n",
        "            tasks=self.tasks,\n",
        "            max_seq_len=self.max_seq_len,\n",
        "            tokenizer=self.tokenizer,\n",
        "        )\n",
        "        return features\n",
        "      \n",
        "      \n",
        "# Helper\n",
        "def read_tsv(filename, rename_columns, quotechar='\"', delimiter=\"\\t\", skiprows=None, header=0, proxies=None, max_samples=None):\n",
        "    \"\"\"Reads a tab separated value file. Tries to download the data if filename is not found\"\"\"\n",
        "    \n",
        "    # get remote dataset if needed\n",
        "    if not (os.path.exists(filename)):\n",
        "        logger.info(f\" Couldn't find {filename} locally. Trying to download ...\")\n",
        "        _download_extract_downstream_data(filename)\n",
        "    \n",
        "    # read file into df\n",
        "    df = pd.read_csv(\n",
        "        filename,\n",
        "        sep=delimiter,\n",
        "        encoding=\"utf-8\",\n",
        "        quotechar=quotechar,\n",
        "        dtype=str,\n",
        "        skiprows=skiprows,\n",
        "        header=header\n",
        "    )\n",
        "\n",
        "    # let's rename our target columns to the default names FARM expects: \n",
        "    # \"text\": contains the text\n",
        "    # \"text_classification_label\": contains a label for text classification\n",
        "    columns = [\"text\"] + list(rename_columns.keys())\n",
        "    df = df[columns]\n",
        "    for source_column, label_name in rename_columns.items():\n",
        "        df[label_name] = df[source_column]\n",
        "        df.drop(columns=[source_column], inplace=True)\n",
        "    \n",
        "    if \"unused\" in df.columns:\n",
        "        df.drop(columns=[\"unused\"], inplace=True)\n",
        "    raw_dict = df.to_dict(orient=\"records\")\n",
        "    return raw_dict"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "05/05/2020 09:28:44 - INFO - transformers.file_utils -   PyTorch version 1.4.0 available.\n",
            "05/05/2020 09:28:45 - INFO - transformers.file_utils -   TensorFlow version 2.2.0-rc3 available.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zjsd5TFDFqzl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "b5fb3077-f0ae-499e-e9f6-9bedb0d6fa7f"
      },
      "source": [
        "# The default format is: \n",
        "# - tab separated\n",
        "# - column \"text\"\n",
        "# - column \"label\" \n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "df = pd.DataFrame({\"text\": [\"The concerts supercaliphractisch was great!\", \"I hate people ignoring climate change.\"],\n",
        "                  \"label\": [\"positive\",\"negative\"]\n",
        "                  })\n",
        "print(df)\n",
        "df.to_csv(\"train.tsv\", sep=\"\\t\")"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                                          text     label\n",
            "0  The concerts supercaliphractisch was great!  positive\n",
            "1       I hate people ignoring climate change.  negative\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F8msjxryFqzv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203,
          "referenced_widgets": [
            "0fedf11acab6486d8c734b79290d6875",
            "daced229491c441ea69ffe85c36867a5",
            "c39407be900946029a6103d448fb8fb2",
            "c990a7513c1a4a3398eaef7699fcda7b",
            "a421ec39b3b944e6bd444ef6019a5808",
            "29c9b4c052f5482097b678a2b74bb626",
            "7dcf072091ac4753a9d2a8f41d5ecc82",
            "2be8f3cd6a81443a8d74f08a98c2b23b"
          ]
        },
        "outputId": "80aed84a-7183-48b4-e932-fc61b049ab0c"
      },
      "source": [
        "tokenizer = Tokenizer.load(\n",
        "    pretrained_model_name_or_path=\"bert-base-uncased\")\n",
        "\n",
        "processor = TextClassificationProcessor(data_dir = \"\", \n",
        "                                        tokenizer=tokenizer,\n",
        "                                        max_seq_len=64,\n",
        "                                        label_list=[\"positive\",\"negative\"],\n",
        "                                        label_column_name=\"label\",\n",
        "                                        metric=\"acc\",\n",
        "                                       )"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "05/05/2020 09:28:53 - INFO - farm.modeling.tokenization -   Loading tokenizer of type 'BertTokenizer'\n",
            "05/05/2020 09:28:53 - INFO - filelock -   Lock 140031920012984 acquired on /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084.lock\n",
            "05/05/2020 09:28:53 - INFO - transformers.file_utils -   https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt not found in cache or force_download set to True, downloading to /root/.cache/torch/transformers/tmpi1hgpklj\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0fedf11acab6486d8c734b79290d6875",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=231508, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "05/05/2020 09:28:53 - INFO - transformers.file_utils -   storing https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt in cache at /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
            "05/05/2020 09:28:53 - INFO - transformers.file_utils -   creating metadata file for /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
            "05/05/2020 09:28:53 - INFO - filelock -   Lock 140031920012984 released on /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084.lock\n",
            "05/05/2020 09:28:53 - INFO - transformers.tokenization_utils -   loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "euqZZc7iFqz3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "c0f90f65-49d8-4a71-e2f8-ff9ee61b65b8"
      },
      "source": [
        "#  1. One File -> Dictionarie(s) with \"raw data\"\n",
        "dicts = processor.file_to_dicts(file=\"train.tsv\")\n",
        "print(dicts)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[{'text': 'The concerts supercaliphractisch was great!', 'text_classification_label': 'positive'}, {'text': 'I hate people ignoring climate change.', 'text_classification_label': 'negative'}]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QVyOl5evFqz9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 517
        },
        "outputId": "14ec7a69-1170-48aa-997a-1176ff2651e3"
      },
      "source": [
        "#  2. One Dictionary -> Sample(s) \n",
        "#     (Sample = \"clear text\" model input + meta information) \n",
        "samples = processor._dict_to_samples(dictionary=dicts[0])\n",
        "# print each attribute of sample\n",
        "print(samples[0].clear_text)\n",
        "print(samples[0].tokenized)\n",
        "print(samples[0].features)\n",
        "print(\"----------------------------------\\n\\n\\n\")\n",
        "# or in a nicer, formatted style\n",
        "print(samples[0])"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'text': 'The concerts supercaliphractisch was great!', 'text_classification_label': 'positive'}\n",
            "{'tokens': ['the', 'concerts', 'super', '##cal', '##ip', '##hra', '##ct', '##isch', 'was', 'great', '!'], 'offsets': [0, 4, 13, 18, 21, 23, 26, 28, 33, 37, 42], 'start_of_word': [True, True, True, False, False, False, False, False, True, True, False]}\n",
            "None\n",
            "----------------------------------\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "      .--.        _____                       _      \n",
            "    .'_\\/_'.     / ____|                     | |     \n",
            "    '. /\\ .'    | (___   __ _ _ __ ___  _ __ | | ___ \n",
            "      \"||\"       \\___ \\ / _` | '_ ` _ \\| '_ \\| |/ _ \\ \n",
            "       || /\\     ____) | (_| | | | | | | |_) | |  __/\n",
            "    /\\ ||//\\)   |_____/ \\__,_|_| |_| |_| .__/|_|\\___|\n",
            "   (/\\||/                             |_|           \n",
            "______\\||/___________________________________________                     \n",
            "\n",
            "ID: None\n",
            "Clear Text: \n",
            " \ttext: The concerts supercaliphractisch was great!\n",
            " \ttext_classification_label: positive\n",
            "Tokenized: \n",
            " \ttokens: ['the', 'concerts', 'super', '##cal', '##ip', '##hra', '##ct', '##isch', 'was', 'great', '!']\n",
            " \toffsets: [0, 4, 13, 18, 21, 23, 26, 28, 33, 37, 42]\n",
            " \tstart_of_word: [True, True, True, False, False, False, False, False, True, True, False]\n",
            "Features: \n",
            " \tNone\n",
            "_____________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-sOWhXIHFq0C",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "96b296a4-69fc-48fd-8912-346defc20aae"
      },
      "source": [
        "# 3. One Sample -> Features\n",
        "#    (Features = \"vectorized\" model input)\n",
        "features = processor._sample_to_features(samples[0])\n",
        "print(features[0])"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'input_ids': [101, 1996, 6759, 3565, 9289, 11514, 13492, 6593, 19946, 2001, 2307, 999, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'padding_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'segment_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'text_classification_label_ids': [0]}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "03hkuHsoFq0J",
        "colab_type": "text"
      },
      "source": [
        "# 2) Hands-On: Adjust it to your dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wCoBmkpOFq0K",
        "colab_type": "text"
      },
      "source": [
        "## Task 1: Use an existing Processor\n",
        "\n",
        "This works if you have:\n",
        "- standard tasks\n",
        "- common file formats \n",
        "\n",
        "**Example: Text classification on CSV with multiple columns**\n",
        "\n",
        "Dataset: GermEval18 (Hatespeech detection)  \n",
        "Format: TSV  \n",
        "Columns: `text coarse_label fine_label`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fCN7UkbGFq0L",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 236
        },
        "outputId": "1f86d316-0522-4a56-b195-a8a82f22034e"
      },
      "source": [
        "# Download dataset\n",
        "from farm.data_handler import utils\n",
        "utils._download_extract_downstream_data(\"germeval18/train.tsv\")\n",
        "!head -n 10 germeval18/train.tsv\n",
        "\n",
        "# TODO: Initialize a processor for the above file by passing the right arguments\n",
        "\n",
        "processor = TextClassificationProcessor(tokenizer=tokenizer,\n",
        "                                        max_seq_len=128,\n",
        "                                        data_dir=\"germeval18\",\n",
        "                                        train_filename=\"train.tsv\",\n",
        "                                        label_list=[\"OTHER\",\"OFFENSE\"],\n",
        "                                        metric=\"acc\",\n",
        "                                        label_column_name=\"coarse_label\"\n",
        "                                        )"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "05/05/2020 09:29:08 - INFO - farm.data_handler.utils -   downloading and extracting file germeval18 to dir /content/FARM\n",
            "100%|██████████| 525101/525101 [00:00<00:00, 1154163.28B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "text\tcoarse_label\tfine_label\n",
            "@corinnamilborn Liebe Corinna, wir würden dich gerne als Moderatorin für uns gewinnen! Wärst du begeisterbar?\tOTHER\tOTHER\n",
            "@Martin28a Sie haben ja auch Recht. Unser Tweet war etwas missverständlich. Dass das BVerfG Sachleistungen nicht ausschließt, kritisieren wir.\tOTHER\tOTHER\n",
            "@ahrens_theo fröhlicher gruß aus der schönsten stadt der welt theo ⚓️\tOTHER\tOTHER\n",
            "@dushanwegner Amis hätten alles und jeden gewählt...nur Hillary wollten sie nicht und eine Fortsetzung von Obama-Politik erst recht nicht..!\tOTHER\tOTHER\n",
            "@spdde kein verläßlicher Verhandlungspartner. Nachkarteln nach den Sondierzngsgesprächen - schickt diese Stümper #SPD in die Versenkung.\tOFFENSE\tINSULT\n",
            "@Dirki_M Ja, aber wo widersprechen die Zahlen denn denen, die im von uns verlinkten Artikel stehen? In unserem Tweet geht es rein um subs. Geschützte. 2017 ist der gesamte Familiennachzug im Vergleich zu 2016 - die Zahlen, die Hr. Brandner bemüht - übrigens leicht rückläufig gewesen.\tOTHER\tOTHER\n",
            "@milenahanm 33 bis 45 habe ich noch gar nicht gelebt und es geht mir am Arsch vorbei was in dieser Zeit geschehen ist. Ich lebe im heute und jetzt und nicht in der Vergangenheit.\tOFFENSE\tPROFANITY\n",
            "@jayxderxmensch @jayxthexhuman Wieso? Was findest du da unklar?\tOTHER\tOTHER\n",
            "@tagesschau Euere AfD Hetze wirkt. Da könnt ihr stolz sein bei #ARD-Fernsehen\tOFFENSE\tABUSE\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aBX3eyjFFq0R",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "acea2635-dd88-4382-ba7d-8d98a696a3e2"
      },
      "source": [
        "# test it\n",
        "dicts = processor.file_to_dicts(file=\"germeval18/train.tsv\")\n",
        "print(dicts[0])\n",
        "assert dicts[0] == {'text': '@corinnamilborn Liebe Corinna, wir würden dich gerne als Moderatorin für uns gewinnen! Wärst du begeisterbar?', 'text_classification_label': 'OTHER'}"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'text': '@corinnamilborn Liebe Corinna, wir würden dich gerne als Moderatorin für uns gewinnen! Wärst du begeisterbar?', 'text_classification_label': 'OTHER'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pnIO1whZFq0X",
        "colab_type": "text"
      },
      "source": [
        "## Task 2: Build your own Processor\n",
        "This works best for:\n",
        "- custom input files\n",
        "- special preprocessing steps\n",
        "- advanced multitask learning \n",
        "\n",
        "**Example: Text classification with JSON as input file** \n",
        "\n",
        "Dataset: [100k Yelp reviews](https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-downstream/yelp_reviews_100k.json) ( [full dataset](https://https://www.yelp.com/dataset/download), [documentation](https://https://www.yelp.com/dataset/documentation/main))\n",
        "\n",
        "Format: \n",
        "\n",
        "``` \n",
        "{\n",
        "...\n",
        "    // integer, star rating\n",
        "    \"stars\": 4,\n",
        "\n",
        "    // string, the review itself\n",
        "    \"text\": \"Great place to hang out after work: the prices are decent, and the ambience is fun. It's a bit loud, but very lively. The staff is friendly, and the food is good. They have a good selection of drinks.\",\n",
        "...\n",
        "}\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gViE3EnNFq0Z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 302
        },
        "outputId": "91b6531b-18bc-4df6-9df5-031878905cf2"
      },
      "source": [
        "# Download dataset\n",
        "!wget https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-downstream/yelp_reviews_100k.json\n",
        "!head -5 yelp_reviews_100k.json"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-05-05 09:29:21--  https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-downstream/yelp_reviews_100k.json\n",
            "Resolving s3.eu-central-1.amazonaws.com (s3.eu-central-1.amazonaws.com)... 52.219.73.135\n",
            "Connecting to s3.eu-central-1.amazonaws.com (s3.eu-central-1.amazonaws.com)|52.219.73.135|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 78383948 (75M) [application/json]\n",
            "Saving to: ‘yelp_reviews_100k.json’\n",
            "\n",
            "yelp_reviews_100k.j 100%[===================>]  74.75M  26.9MB/s    in 2.8s    \n",
            "\n",
            "2020-05-05 09:29:24 (26.9 MB/s) - ‘yelp_reviews_100k.json’ saved [78383948/78383948]\n",
            "\n",
            "{\"review_id\":\"Q1sbwvVQXV2734tPgoKj4Q\",\"user_id\":\"hG7b0MtEbXx5QzbzE6C_VA\",\"business_id\":\"ujmEBvifdJM6h6RLv4wQIg\",\"stars\":1.0,\"useful\":6,\"funny\":1,\"cool\":0,\"text\":\"Total bill for this horrible service? Over $8Gs. These crooks actually had the nerve to charge us $69 for 3 pills. I checked online the pills can be had for 19 cents EACH! Avoid Hospital ERs at all costs.\",\"date\":\"2013-05-07 04:34:36\"}\n",
            "{\"review_id\":\"GJXCdrto3ASJOqKeVWPi6Q\",\"user_id\":\"yXQM5uF2jS6es16SJzNHfg\",\"business_id\":\"NZnhc2sEQy3RmzKTZnqtwQ\",\"stars\":5.0,\"useful\":0,\"funny\":0,\"cool\":0,\"text\":\"I *adore* Travis at the Hard Rock's new Kelly Cardenas Salon!  I'm always a fan of a great blowout and no stranger to the chains that offer this service; however, Travis has taken the flawless blowout to a whole new level!  \\n\\nTravis's greets you with his perfectly green swoosh in his otherwise perfectly styled black hair and a Vegas-worthy rockstar outfit.  Next comes the most relaxing and incredible shampoo -- where you get a full head message that could cure even the very worst migraine in minutes --- and the scented shampoo room.  Travis has freakishly strong fingers (in a good way) and use the perfect amount of pressure.  That was superb!  Then starts the glorious blowout... where not one, not two, but THREE people were involved in doing the best round-brush action my hair has ever seen.  The team of stylists clearly gets along extremely well, as it's evident from the way they talk to and help one another that it's really genuine and not some corporate requirement.  It was so much fun to be there! \\n\\nNext Travis started with the flat iron.  The way he flipped his wrist to get volume all around without over-doing it and making me look like a Texas pagent girl was admirable.  It's also worth noting that he didn't fry my hair -- something that I've had happen before with less skilled stylists.  At the end of the blowout & style my hair was perfectly bouncey and looked terrific.  The only thing better?  That this awesome blowout lasted for days! \\n\\nTravis, I will see you every single time I'm out in Vegas.  You make me feel beauuuutiful!\",\"date\":\"2017-01-14 21:30:33\"}\n",
            "{\"review_id\":\"2TzJjDVDEuAW6MR5Vuc1ug\",\"user_id\":\"n6-Gk65cPZL6Uz8qRm3NYw\",\"business_id\":\"WTqjgwHlXbSFevF32_DJVw\",\"stars\":5.0,\"useful\":3,\"funny\":0,\"cool\":0,\"text\":\"I have to say that this office really has it together, they are so organized and friendly!  Dr. J. Phillipp is a great dentist, very friendly and professional.  The dental assistants that helped in my procedure were amazing, Jewel and Bailey helped me to feel comfortable!  I don't have dental insurance, but they have this insurance through their office you can purchase for $80 something a year and this gave me 25% off all of my dental work, plus they helped me get signed up for care credit which I knew nothing about before this visit!  I highly recommend this office for the nice synergy the whole office has!\",\"date\":\"2016-11-09 20:09:03\"}\n",
            "{\"review_id\":\"yi0R0Ugj_xUx_Nek0-_Qig\",\"user_id\":\"dacAIZ6fTM6mqwW5uxkskg\",\"business_id\":\"ikCg8xy5JIg_NGPx-MSIDA\",\"stars\":5.0,\"useful\":0,\"funny\":0,\"cool\":0,\"text\":\"Went in for a lunch. Steak sandwich was delicious, and the Caesar salad had an absolutely delicious dressing, with a perfect amount of dressing, and distributed perfectly across each leaf. I know I'm going on about the salad ... But it was perfect.\\n\\nDrink prices were pretty good.\\n\\nThe Server, Dawn, was friendly and accommodating. Very happy with her.\\n\\nIn summation, a great pub experience. Would go again!\",\"date\":\"2018-01-09 20:56:38\"}\n",
            "{\"review_id\":\"11a8sVPMUFtaC7_ABRkmtw\",\"user_id\":\"ssoyf2_x0EQMed6fgHeMyQ\",\"business_id\":\"b1b1eb3uo-w561D0ZfCEiQ\",\"stars\":1.0,\"useful\":7,\"funny\":0,\"cool\":0,\"text\":\"Today was my second out of three sessions I had paid for. Although my first session went well, I could tell Meredith had a particular enjoyment for her male clients over her female. However, I returned because she did my teeth fine and I was pleased with the results. When I went in today, I was in the whitening room with three other gentlemen. My appointment started out well, although, being a person who is in the service industry, I always attend to my female clientele first when a couple arrives. Unbothered by those signs, I waited my turn. She checked on me once after my original 30 minute timer to ask if I was ok. She attended my boyfriend on numerous occasions, as well as the other men, and would exit the room without even asking me or looking to see if I had any irritation. Half way through, another woman had showed up who she was explaining the deals to in the lobby. While she admits timers must be reset half way through the process, she reset my boyfriends, left, rest the gentleman furthest away from me who had time to come in, redeem his deal, get set, and gave his timer done, before me, then left, and at this point my time was at 10 minutes. So, she should have reset it 5 minutes ago, according to her. While I sat there patiently this whole time with major pain in my gums, i watched the time until the lamp shut off. Not only had she reset two others, explained deals to other guest, but she never once checked on my time. When my light turned off, I released the stance of my mouth to a more relaxed state, assuming I was only getting a thirty minute session instead of the usual 45, because she had yet to come in. At this point, the teeth formula was not only burning the gum she neglected for 25 minutes now, but it began to burn my lips. I began squealing and slapping my chair trying to get her attention from the other room in a panic. I was in so much pain, that by the time she entered the room I was already out of my chair. She finally then acknowledged me, and asked if she could put vitamin E on my gum burn (pictured below). At this point, she has treated two other gums burns, while neglecting me, and I was so irritated that I had to suffer, all I wanted was to leave. While I waited for my boyfriend, she kept harassing me about the issue. Saying, \\\"well burns come with teeth whitening.\\\" While I totally agree, and under justifiable circumstances would not be as irritate, it could have easily been avoid if she had checked on me even a second time, so I could let her know. Not only did she never check on my physical health, she couldn't even take two seconds to reset the timer, which she even admitted to me. Her accuse was that she was coming in to do it, but I had the light off for a solid two minutes before I couldn't stand the pain. She admitted it should be reset every 15 minutes, which means for 25 minutes she did not bother to help me at all. Her guest in the lobby then proceeded to attack me as well, simply because I wanted to leave after the way I was treated. I also expected a refund for not getting a complete session today, due to the neglect, and the fact I won't be returning for my last, she had failed to do that. She was even screaming from the door, and continued to until my boyfriend and I were down the steps. I have never in my life been more appalled by a grown woman's behavior, who claims to be in the business for \\\"10 years.\\\" Admit your wrongs, but don't make your guest feel unwelcome because you can't do you job properly.\",\"date\":\"2018-01-30 23:07:38\"}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BHtzq11EFq0d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "# TODO: Create a new Processor class and overwrite the function that reads from the file\n",
        "# The dicts created should look like this to comply with the default TextClassificationProcessor.\n",
        "#{'text': 'Total bill for this horrible service? ...',\n",
        "# 'text_classification_label': '4'}\n",
        "\n",
        "\n",
        "class CustomTextClassificationProcessor(TextClassificationProcessor):\n",
        "  \n",
        "    # we need to overwrite this function from the parent class\n",
        "    def file_to_dicts(self, file: str) -> [dict]:\n",
        "      # read into df\n",
        "      df = pd.read_json(file, lines=True)\n",
        "      # rename\n",
        "      df[\"text_classification_label\"] = df[\"stars\"].astype(str)\n",
        "      # drop unused\n",
        "      columns = [\"text_classification_label\",\"text\"]\n",
        "      df = df[columns]\n",
        "      # convert to dicts\n",
        "      dicts = df.to_dict(orient=\"records\")\n",
        "      return dicts"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ovkucFdrFq0k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "processor = CustomTextClassificationProcessor(tokenizer=tokenizer,\n",
        "                                              max_seq_len=128,\n",
        "                                              data_dir=\"\",\n",
        "                                              label_list=[\"1\",\"2\",\"3\",\"4\",\"5\"],\n",
        "                                              metric=\"acc\",\n",
        "                                              )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kfy9RsbWFq0o",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "3883887a-d931-497d-8ab9-e1024e039207"
      },
      "source": [
        "# test it\n",
        "\n",
        "dicts = processor.file_to_dicts(file=\"yelp_reviews_100k.json\")\n",
        "print(dicts[0])\n",
        "\n",
        "assert dicts[0] == {'text_classification_label': '1', 'text': 'Total bill for this horrible service? Over $8Gs. These crooks actually had the nerve to charge us $69 for 3 pills. I checked online the pills can be had for 19 cents EACH! Avoid Hospital ERs at all costs.'}"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'text_classification_label': '1', 'text': 'Total bill for this horrible service? Over $8Gs. These crooks actually had the nerve to charge us $69 for 3 pills. I checked online the pills can be had for 19 cents EACH! Avoid Hospital ERs at all costs.'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ga1r_ZgbFq0t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}